import re

import requests
from lxml import etree

headers = {
    "Referer": "https://kns.cnki.net/kns8s/defaultresult/index?korder=&kw=",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
}
url = "https://kns.cnki.net/kns8s/brief/grid"
page = 300
while True:
    page += 1
    data = {
        "boolSearch": "false",
        "QueryJson": "{\"Platform\":\"\",\"Resource\":\"CROSSDB\",\"Classid\":\"WD0FTY92\",\"Products\":\"\",\"QNode\":{\"QGroup\":[{\"Key\":\"Subject\",\"Title\":\"\",\"Logic\":0,\"Items\":[{\"Field\":\"SU\",\"Value\":\"爬虫\",\"Operator\":\"TOPRANK\",\"Logic\":0,\"Title\":\"主题\"}],\"ChildItems\":[]}]},\"ExScope\":1,\"SearchType\":2,\"Rlang\":\"CHINESE\",\"KuaKuCode\":\"YSTT4HG0,LSTPFY1C,JUP3MUPD,MPMFIG1A,EMRPGLPA,WQ0UVIAA,BLZOG7CK,PWFIRAGL,NN3FJMUV,NLBO1Z6R\",\"SearchFrom\":4}",
        "pageNum": page,
        "pageSize": "20",
        "sortField": "PT",
        "sortType": "desc",
        "dstyle": "listmode",
        "boolSortSearch": "false",
        "productStr": "YSTT4HG0,LSTPFY1C,RMJLXHZ3,JQIRZIYA,JUP3MUPD,1UR4K4HZ,BPBAFJ5S,R79MZMCB,MPMFIG1A,EMRPGLPA,J708GVCE,ML4DRIDX,WQ0UVIAA,NB3BWEHK,XVLO76FD,HR1YT1Z9,BLZOG7CK,PWFIRAGL,NN3FJMUV,NLBO1Z6R,",
        "aside": "",
        "searchFrom": "资源范围：总库"
    }
    response = requests.post(url, headers=headers, data=data)
    html_str = response.content.decode()
    print(response.status_code)
    print(html_str)
    root = etree.HTML(html_str)
    tr_list = root.xpath("//table[@class='result-table-list']/tbody/tr")
    print(len(tr_list))
    for tr in tr_list:
        title = "".join(tr.xpath(".//td[@class='name']//text()"))
        title = re.sub("\s+", "", title)
        author = tr.xpath(".//td[@class='author']/a[@class='KnowledgeNetLink']/text()")
        print(page, title, author)
    break
